package data.crawling;

import java.net.MalformedURLException;

import data.aws.AmazonItemSearch;
import websphinx.Crawler;
import websphinx.Link;
import websphinx.StandardClassifier;
import websphinx.workbench.Statistics;

public class RunMyCrawler {
		
//	private final static String [] mans = {
//			"Agfa", 
//			"Canon", 
//			"Casio", 
//			"Contax", 
//			"Epson", 
//			"FujiFilm", 
//			"HP", 
//			"Kodak", 
//			"Konica_Minolta", 
//			"Kyocera", 
//			"Leica", 
//			"Nikon", 
//			"Olympus", 
//			"Panasonic", 
//			"Pentax", 
//			"Ricoh", 
//			"Samsung", 
//			"Sanyo", 
//			"Sigma", 
//			"Sony", 
//			"Toshiba",
//			};
	
	public static void main(String[] args) throws MalformedURLException {
		String MANUFACTURER = "";

		String[] mans = {"olympus"};
		int i = 0;
		Link[] rootlinks;
		int maxDepth = Builder.maxCrawlDepth;
		PageCrawler pc = new PageCrawler();
		while(i<mans.length){		
			MANUFACTURER = mans[i++];
			pc.setCurrentMan(MANUFACTURER);
			if(MANUFACTURER.contains("Konica")){
				MANUFACTURER = MANUFACTURER.replace("_", "");
			}
			if(Builder.buildAssertions){
				rootlinks = new Link[2];
				String rooturl1 = "http://www.dpreview.com/products/"+MANUFACTURER.toLowerCase()+"/compacts/";
				String rooturl2 = "http://www.dpreview.com/products/"+MANUFACTURER.toLowerCase()+"/slrs/";
				rootlinks[0] = new Link(rooturl1);
				rootlinks[1] = new Link(rooturl2);
			}else{
				String rooturl = "http://www.dpreview.com/learn/?";
				rootlinks = new Link[1];
				rootlinks[0] = new Link(rooturl);
				i = mans.length;
			}
			
			pc.setRoots(rootlinks);
			//use standard classifier for pages
			pc.addClassifier(new StandardClassifier());
			pc.setMaxDepth(maxDepth);
			pc.setDepthFirst(true);
			//only visits subtree of the given link
			pc.setDomain(Crawler.SUBTREE);
			pc.addCrawlListener(new Statistics());
			//only hyperlinks are expanded.
			pc.setLinkType(Crawler.HYPERLINKS);
			pc.clear();
			pc.run();
			//wait for 30 secs to aoivd Google's complaints~~~
//			try {
//				Thread.sleep(1000);
//			} catch (InterruptedException e) {
//				e.printStackTrace();
//			}
		}

		//cache hit for sellers
		System.out.println("Seller cache hit times: "+AmazonItemSearch.countcache);
		//start build the KB after all pages are visited
//		if(pc.getPagesLeft() <1){
////			if(Builder.manualTBox){
////				TBoxBuilder tbd = new TBoxBuilder(Builder.fileName);
////				tbd.buildTBox();
////			}		
////			if(Builder.buildTerm){		
////				//TermBuilder tbd = new TermBuilder(Builder.fileName);
////				//tbd.buildTerms(pc.terms);		
////			}
//			AssertionBuilder abd = new AssertionBuilder();
//			if(Builder.buildAssertions ){
////				System.out.println("cam size:"+pc.getCameras().size());
////				abd.buildAssertions(pc.getCameras());
//			}
//		}
		//
	}

}
